AirFlow


  • Notes
    # logs for specic sub process
    kail -n airflow -c sync-repositories
    
    # login to airflow-web:sync-repositories pod
    . /opt/bitnami/scripts/git/entrypoint.sh; \
    cd /dags/x-flow && git pull origin develop 
    
    for c in $(k get po |cut -d ' ' -f 1 |egrep 'sch|web|work'); do \
    echo $c; k exec -it $c -c sync-repositories -- /bin/bash -c \
    '. /opt/bitnami/scripts/git/entrypoint.sh ; cd /dags/x-flow && git pull origin develop'\; \
    done
    
    # logs on scheduler
    /opt/bitnami/airflow/logs
    
    ssh-keyscan -t ed25519 bitbucket.org > ./mro/ssh/known_hosts
    ssh-keygen -y -f ./mro/ssh/x-flow-key.pem > ./mro/ssh/id.pub
    kubectl create secret generic x-flow-ssh-key-secret \
     --from-file=id_rsa=./mro/ssh/x-flow-key.pem \
     --from-file=known_hosts=./mro/ssh/known_hosts \
     --from-file=id_rsa.pub=./mro/ssh/id.pub \
     -n airflow
    
        extraEnvVars:
          - name: GIT_SSH_COMMAND
            value: "ssh -i /opt/bitnami/git/.ssh/id_rsa -o StrictHostKeyChecking=no"
    
    
    /Users/myron/.asdf/installs/krew/0.4.4/bin/kubectl-resource_capacity --pods |grep air
    ip-172-31-28-33   airflow airflow-redis-master-0        100m (1%)      150m (1%)       128Mi (0%)        192Mi (0%)
    ip-172-31-28-33   airflow airflow-scheduler-6cx8-swk7p  600m (7%)      900m (11%)     1152Mi (3%)       1728Mi (5%)
    ip-172-31-28-33   airflow airflow-web-69x4-n5xqf       1100m (13%)    1650m (20%)     2176Mi (6%)       3264Mi (10%)
    ip-172-31-28-33   airflow airflow-worker-0             1100m (13%)    1650m (20%)     2176Mi (6%)       3264Mi (10%)
  • More notes
    docker.io/bitnami/airflow-scheduler:2.10.2-debian-12-r0
    
    adduser/now 3.134 all [installed,local]
    apt/now 2.6.1 amd64 [installed,local]
    base-files/now 12.4+deb12u7 amd64 [installed,local]
    base-passwd/now 3.6.1 amd64 [installed,local]
    bash/now 5.2.15-2+b7 amd64 [installed,local]
    bsdutils/now 1:2.38.1-5+deb12u1 amd64 [installed,local]
    ca-certificates/now 20230311 all [installed,local]
    comerr-dev/now 2.1-1.47.0-2 amd64 [installed,local]
    coreutils/now 9.1-1 amd64 [installed,local]
    dash/now 0.5.12-2 amd64 [installed,local]
    debconf/now 1.5.82 all [installed,local]
    debian-archive-keyring/now 2023.3+deb12u1 all [installed,local]
    debianutils/now 5.7-0.5~deb12u1 amd64 [installed,local]
    diffutils/now 1:3.8-4 amd64 [installed,local]
    dpkg/now 1.21.22 amd64 [installed,local]
    findutils/now 4.9.0-4 amd64 [installed,local]
    freetds-common/now 1.3.17+ds-2 all [installed,local]
    gcc-12-base/now 12.2.0-14 amd64 [installed,local]
    gpgv/now 2.2.40-1.1 amd64 [installed,local]
    grep/now 3.8-5 amd64 [installed,local]
    gzip/now 1.12-1 amd64 [installed,local]
    hostname/now 3.23+nmu1 amd64 [installed,local]
    init-system-helpers/now 1.65.2 all [installed,local]
    insserv/now 1.24.0-1 amd64 [installed,local]
    krb5-multidev/now 1.20.1-2+deb12u2 amd64 [installed,local]
    krb5-user/now 1.20.1-2+deb12u2 amd64 [installed,local]
    libacl1/now 2.3.1-3 amd64 [installed,local]
    libapt-pkg6.0/now 2.6.1 amd64 [installed,local]
    libattr1/now 1:2.5.1-4 amd64 [installed,local]
    libaudit-common/now 1:3.0.9-1 all [installed,local]
    libaudit1/now 1:3.0.9-1 amd64 [installed,local]
    libblkid1/now 2.38.1-5+deb12u1 amd64 [installed,local]
    libbsd0/now 0.11.7-2 amd64 [installed,local]
    libbz2-1.0/now 1.0.8-5+b1 amd64 [installed,local]
    libc-bin/now 2.36-9+deb12u8 amd64 [installed,local]
    libc-dev-bin/now 2.36-9+deb12u8 amd64 [installed,local]
    libc-l10n/now 2.36-9+deb12u8 all [installed,local]
    libc6-dev/now 2.36-9+deb12u8 amd64 [installed,local]
    libc6/now 2.36-9+deb12u8 amd64 [installed,local]
    libcap-ng0/now 0.8.3-1+b3 amd64 [installed,local]
    libcap2/now 1:2.66-4 amd64 [installed,local]
    libcom-err2/now 1.47.0-2 amd64 [installed,local]
    libcrypt-dev/now 1:4.4.33-2 amd64 [installed,local]
    libcrypt1/now 1:4.4.33-2 amd64 [installed,local]
    libdb5.3/now 5.3.28+dfsg2-1 amd64 [installed,local]
    libdebconfclient0/now 0.270 amd64 [installed,local]
    libedit2/now 3.1-20221030-2 amd64 [installed,local]
    libevent-2.1-7/now 2.1.12-stable-8 amd64 [installed,local]
    libffi8/now 3.4.4-1 amd64 [installed,local]
    libfile-find-rule-perl/now 0.34-3 all [installed,local]
    libgcc-s1/now 12.2.0-14 amd64 [installed,local]
    libgcrypt20/now 1.10.1-3 amd64 [installed,local]
    libgdbm-compat4/now 1.23-3 amd64 [installed,local]
    libgdbm6/now 1.23-3 amd64 [installed,local]
    libgmp-dev/now 2:6.2.1+dfsg1-1.1 amd64 [installed,local]
    libgmp10/now 2:6.2.1+dfsg1-1.1 amd64 [installed,local]
    libgmpxx4ldbl/now 2:6.2.1+dfsg1-1.1 amd64 [installed,local]
    libgnutls-dane0/now 3.7.9-2+deb12u3 amd64 [installed,local]
    libgnutls-openssl27/now 3.7.9-2+deb12u3 amd64 [installed,local]
    libgnutls28-dev/now 3.7.9-2+deb12u3 amd64 [installed,local]
    libgnutls30/now 3.7.9-2+deb12u3 amd64 [installed,local]
    libgnutlsxx30/now 3.7.9-2+deb12u3 amd64 [installed,local]
    libgpg-error0/now 1.46-1 amd64 [installed,local]
    libgss-dev/now 1.0.4-1 amd64 [installed,local]
    libgss3/now 1.0.4-1 amd64 [installed,local]
    libgssapi-krb5-2/now 1.20.1-2+deb12u2 amd64 [installed,local]
    libgssrpc4/now 1.20.1-2+deb12u2 amd64 [installed,local]
    libhogweed6/now 3.8.1-2 amd64 [installed,local]
    libicu72/now 72.1-3 amd64 [installed,local]
    libidn-dev/now 1.41-1 amd64 [installed,local]
    libidn11-dev/now 1.41-1 amd64 [installed,local]
    libidn12/now 1.41-1 amd64 [installed,local]
    libidn2-0/now 2.3.3-1+b1 amd64 [installed,local]
    libidn2-dev/now 2.3.3-1+b1 amd64 [installed,local]
    libk5crypto3/now 1.20.1-2+deb12u2 amd64 [installed,local]
    libkadm5clnt-mit12/now 1.20.1-2+deb12u2 amd64 [installed,local]
    libkadm5srv-mit12/now 1.20.1-2+deb12u2 amd64 [installed,local]
    libkdb5-10/now 1.20.1-2+deb12u2 amd64 [installed,local]
    libkeyutils1/now 1.6.3-2 amd64 [installed,local]
    libkrb5-3/now 1.20.1-2+deb12u2 amd64 [installed,local]
    libkrb5-dev/now 1.20.1-2+deb12u2 amd64 [installed,local]
    libkrb5support0/now 1.20.1-2+deb12u2 amd64 [installed,local]
    libldap-2.5-0/now 2.5.13+dfsg-5 amd64 [installed,local]
    liblz4-1/now 1.9.4-1 amd64 [installed,local]
    liblzma5/now 5.4.1-0.2 amd64 [installed,local]
    libmariadb3/now 1:10.11.6-0+deb12u1 amd64 [installed,local]
    libmd0/now 1.0.4-2 amd64 [installed,local]
    libmount1/now 2.38.1-5+deb12u1 amd64 [installed,local]
    libncursesw6/now 6.4-4 amd64 [installed,local]
    libnettle8/now 3.8.1-2 amd64 [installed,local]
    libnsl-dev/now 1.3.0-2 amd64 [installed,local]
    libnsl2/now 1.3.0-2 amd64 [installed,local]
    libnumber-compare-perl/now 0.03-3 all [installed,local]
    libp11-kit-dev/now 0.24.1-2 amd64 [installed,local]
    libp11-kit0/now 0.24.1-2 amd64 [installed,local]
    libpam-modules-bin/now 1.5.2-6+deb12u1 amd64 [installed,local]
    libpam-modules/now 1.5.2-6+deb12u1 amd64 [installed,local]
    libpam-runtime/now 1.5.2-6+deb12u1 all [installed,local]
    libpam0g/now 1.5.2-6+deb12u1 amd64 [installed,local]
    libpcre2-8-0/now 10.42-1 amd64 [installed,local]
    libperl5.36/now 5.36.0-7+deb12u1 amd64 [installed,local]
    libpkgconf3/now 1.8.1-1 amd64 [installed,local]
    libproc2-0/now 2:4.0.2-3 amd64 [installed,local]
    libreadline8/now 8.2-1.3 amd64 [installed,local]
    libsasl2-2/now 2.1.28+dfsg-10 amd64 [installed,local]
    libsasl2-modules-db/now 2.1.28+dfsg-10 amd64 [installed,local]
    libsasl2-modules/now 2.1.28+dfsg-10 amd64 [installed,local]
    libseccomp2/now 2.5.4-1+deb12u1 amd64 [installed,local]
    libselinux1/now 3.4-1+b6 amd64 [installed,local]
    libsemanage-common/now 3.4-1 all [installed,local]
    libsemanage2/now 3.4-1+b5 amd64 [installed,local]
    libsepol2/now 3.4-2.1 amd64 [installed,local]
    libshishi-dev/now 1.0.3-1+b1 amd64 [installed,local]
    libshishi0/now 1.0.3-1+b1 amd64 [installed,local]
    libsmartcols1/now 2.38.1-5+deb12u1 amd64 [installed,local]
    libsqlite3-0/now 3.40.1-2 amd64 [installed,local]
    libss2/now 1.47.0-2 amd64 [installed,local]
    libssl3/now 3.0.14-1~deb12u2 amd64 [installed,local]
    libstdc++6/now 12.2.0-14 amd64 [installed,local]
    libsybdb5/now 1.3.17+ds-2 amd64 [installed,local]
    libsystemd0/now 252.30-1~deb12u2 amd64 [installed,local]
    libtasn1-6-dev/now 4.19.0-2 amd64 [installed,local]
    libtasn1-6/now 4.19.0-2 amd64 [installed,local]
    libtext-glob-perl/now 0.11-3 all [installed,local]
    libtinfo6/now 6.4-4 amd64 [installed,local]
    libtirpc-common/now 1.3.3+ds-1 all [installed,local]
    libtirpc-dev/now 1.3.3+ds-1 amd64 [installed,local]
    libtirpc3/now 1.3.3+ds-1 amd64 [installed,local]
    libudev1/now 252.30-1~deb12u2 amd64 [installed,local]
    libunbound8/now 1.17.1-2+deb12u2 amd64 [installed,local]
    libunistring2/now 1.0-2 amd64 [installed,local]
    libuuid1/now 2.38.1-5+deb12u1 amd64 [installed,local]
    libxml2/now 2.9.14+dfsg-1.3~deb12u1 amd64 [installed,local]
    libxslt1.1/now 1.1.35-1 amd64 [installed,local]
    libxxhash0/now 0.8.1-1 amd64 [installed,local]
    libzstd1/now 1.5.4+dfsg2-5 amd64 [installed,local]
    linux-libc-dev/now 6.1.106-3 amd64 [installed,local]
    locales/now 2.36-9+deb12u8 all [installed,local]
    login/now 1:4.13+dfsg1-1+b1 amd64 [installed,local]
    lsb-base/now 11.6 all [installed,local]
    mariadb-common/now 1:10.11.6-0+deb12u1 all [installed,local]
    mawk/now 1.3.4.20200120-3.1 amd64 [installed,local]
    mount/now 2.38.1-5+deb12u1 amd64 [installed,local]
    mysql-common/now 5.8+1.1.0 all [installed,local]
    ncurses-base/now 6.4-4 all [installed,local]
    netbase/now 6.4 all [installed,local]
    nettle-dev/now 3.8.1-2 amd64 [installed,local]
    openssl/now 3.0.14-1~deb12u2 amd64 [installed,local]
    passwd/now 1:4.13+dfsg1-1+b1 amd64 [installed,local]
    perl-base/now 5.36.0-7+deb12u1 amd64 [installed,local]
    perl-modules-5.36/now 5.36.0-7+deb12u1 all [installed,local]
    perl/now 5.36.0-7+deb12u1 amd64 [installed,local]
    pkg-config/now 1.8.1-1 amd64 [installed,local]
    pkgconf-bin/now 1.8.1-1 amd64 [installed,local]
    pkgconf/now 1.8.1-1 amd64 [installed,local]
    procps/now 2:4.0.2-3 amd64 [installed,local]
    readline-common/now 8.2-1.3 all [installed,local]
    rpcsvc-proto/now 1.4.3-1 amd64 [installed,local]
    sed/now 4.9-1 amd64 [installed,local]
    sgml-base/now 1.31 all [installed,local]
    shishi-common/now 1.0.3-1 all [installed,local]
    startpar/now 0.65-1+b1 amd64 [installed,local]
    sysv-rc/now 3.06-4 all [installed,local]
    sysvinit-utils/now 3.06-4 amd64 [installed,local]
    tar/now 1.34+dfsg-1.2+deb12u1 amd64 [installed,local]
    tzdata/now 2024a-0+deb12u1 all [installed,local]
    usrmerge/now 37~deb12u1 all [installed,local]
    util-linux-extra/now 2.38.1-5+deb12u1 amd64 [installed,local]
    util-linux/now 2.38.1-5+deb12u1 amd64 [installed,local]
    zlib1g/now 1:1.2.13.dfsg-1 amd64 [installed,local]
  • AWS Metrics
    # from container insights
    SELECT AVG(ContainerMemoryUtilization) FROM SCHEMA("ECS/ContainerInsights", ClusterName,TaskDefinitionFamily,TaskId,ContainerName) WHERE ClusterName='batch-prod_seg-ce-32_Batch_68795af3-1dd6-30df-8a02-3546ad81c16c' AND AWS.AccountId = '556647978552' GROUP BY ContainerName, TaskId ORDER BY MAX() DESC LIMIT 10